package top.timestreet.xmsk.timestreet.crawl;

import java.util.ArrayList;
import java.util.List;

import top.timestreet.xmsk.timestreet.pojo.ListPage;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.model.OOSpider;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class ListPageProcessor implements PageProcessor{

	private Site site = Site.me();
	
	@Override
	public void process(Page page) {
		List<ListPage> lists = new ArrayList<>();
		System.out.println("进入了ListPageProcessor");
		page.putField("title",page);
		// 取到每一个循环的列表项
		List<Selectable> xpath = (List<Selectable>) page.getHtml().xpath("//*[@id=\"main\"]/div/div/div/article").nodes();
		for (int i = 0; i < xpath.size(); i++) {
			Selectable selectable = xpath.get(i);
			ListPage listPage = new ListPage();
			// 取出每一个列表项里的标题，摘要
			listPage.setTitle(selectable.$(".article-title a", "text").toString().trim());
			listPage.setReadme(selectable.$(".article-text p").toString().trim());
			lists.add(listPage);
		}
		// 添加的page里面，在Pipline里面根据 page的 key = listPage 取出 value = lists
		page.putField("listPage", lists);
		// 抓取下一页，抓两页就够了
		page.addTargetRequest("http://www.timestreet.top/page/2");
	}

	@Override
	public Site getSite() {
		return site;
	}
	
	public static void main(String[] args) {
		OOSpider.create(new ListPageProcessor())
			.addUrl("http://www.timestreet.top/page/1")
			.addPipeline((Pipeline) new ListPagePipeline())
			.setExitWhenComplete(true)
			.start();
	}

}
